package com.guchenbo.spark.core

import org.apache.spark.SparkContext

/**
 * @author guchenbo
 * @date 2022/3/11
 */
object RddSrc {

  def main(args: Array[String]): Unit = {
    val sc = SparkUtils.sc("rdd src", "local")

    //    noShuffle(sc)
    val f = (s: Int) => {
      println(s)
    }
    val mr = () => sc.makeRDD(List(1, 2, 3, 4, 5, 6))

    val shuffle = () => {
      val rdd = mr()
      val rdd2 = rdd.map((_, 1))
      val rdd3 = rdd2.filter(_._1 != 5)
      val rdd4 = rdd3.reduceByKey(_ + _)
      rdd4.collect().foreach(println)
    }

    val noShuffle = () => {
      val rdd = mr()
      val rdd2 = rdd.map((_, 1))
      val rdd3 = rdd2.filter(_._1 != 5)
      rdd3.collect().foreach(println)
      println(rdd3.count())
    }

    noShuffle()

  }


}
